#coding:utf-8

import json

def count(filename1,filename2,filename3):
    zC = dict()
    vC = dict()
    with open(filename1,mode='r') as f:
        for line in f:
            _,zs,vs,_ = json.loads(line)
            zzs = set()
            for v,zz in zs:
                zzs.update(zz)
            for z in zzs:
                c = zC.get(z,0)
                zC[z] = c+1
            for v,_ in vs.items():
                c = vC.get(v,0)
                vC[v] = c+1
    print len(zC),len(vC)
    zCs = [ (z,c) for z,c in zC.items() ]
    vCs = [ (v,c) for v,c in vC.items() ]
    zCs.sort(key = lambda x : -x[1])
    vCs.sort(key = lambda x : -x[1])
    with open(filename2,mode='w') as f:
        for z,c in zCs:
            line = json.dumps((z,c),ensure_ascii=False)
            f.write(line.encode('utf-8')+'\n')
    with open(filename3,mode='w') as f:
        for v,c in vCs:
            line = json.dumps((v,c),ensure_ascii=False)
            f.write(line.encode('utf-8')+'\n')


def getTheDict(cc,filename1,filename2):
    f1 = open(filename1,mode='r')
    f2 = open(filename2,mode='w')

    m = dict()
    for line in f1:
        v,c = json.loads(line)
        if cc < c:
            m[v] = len(m)

    ms = [ (v,c) for v,c in m.items() ]
    ms.sort(key = lambda x : x[1])

    for v,c in ms:
        line = json.dumps((v,c),ensure_ascii=False)
        f2.write(line.encode('utf-8')+"\n")
    
    f1.close()
    f2.close()


def initDict(filename):
    M = dict()
    with open(filename,mode='r') as f:
        for line in f:
            z,c = json.loads(line)
            M[z] = c
    return M

def saveDict(M,filename):
    ms = [ (v,c) for v,c in M.items() ]
    ms.sort(key = lambda x : x[1])
    f = open(filename,mode='w')
    for v,c in ms:
        line = json.dumps((v,c),ensure_ascii=False)
        f.write(line.encode('utf-8')+"\n")  
    f.close()  

def insertNone(cc,filename1,filename2):
    M = initDict(filename1)
    for i in range(cc):
        n = 'None_%s'%i
        M[n] = len(M)
    saveDict(M,filename2)

if __name__ == '__main__':
    
    print 'Hello'

    filename1 = 'data.2.txt'
    filename2 = 'zCount.txt'
    filename3 = 'vCount.txt'
    #count(filename1,filename2,filename3)

    filename21 = 'zCount.txt'
    filename22 = 'zDict.txt'
    getTheDict(50,filename21,filename22) 

    filename23 = 'zDict.none.txt'
    insertNone(5,filename22,filename23)

    filename31 = 'vCount.txt'
    filename32 = 'vDict.txt'
    getTheDict(20,filename31,filename32)

